package com.yx.crawler.typedef;

import java.net.URI;
import java.net.URL;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.jsoup.Connection;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.yx.crawler.analyse.SimilarityAnalyse;
import com.yx.crawler.analyse.SimilarityAnalyse.Similarity;
import com.yx.crawler.analyse.SimilarityModel;
import com.yx.crawler.util.ContentExtractor;
import com.yx.crawler.util.ContentExtractor.Content;
import com.yx.crawler.util.TextExtract;

public class CrawlerPage {
	private CrawlerUrl currentCrawlerUrl;
	private Document document = null;
	private Date lastModified = null;
	private int length = 0;
	private String contentType = "text/html";
	private String content = null;

	public CrawlerPage(CrawlerUrl crawlerUrl, Connection.Response response) throws Exception {
		this.currentCrawlerUrl = crawlerUrl;
		SimpleDateFormat sdf = new SimpleDateFormat("EEE',' dd MMM yyyy HH:mm:ss 'GMT'", Locale.ENGLISH);
		contentType = response.contentType();
		try {
			lastModified = sdf.parse(response.header("Last-Modified"));
		} catch (Exception e) {
			lastModified = new Date();
		}
		try {
			length = Integer.parseInt(response.header("Content-Length"));
		} catch (Exception e) {
		}
		this.document = response.parse();
	}

	public CrawlerUrl getCrawlerUrl() {
		return currentCrawlerUrl;
	}

	public List<CrawlerUrl> getChildUrls(Element doc, CrawlerListener childCrawlerListener,
			CrawlerUrlFilter crawlerUrlFilter) throws Exception {
		List<CrawlerUrl> crawlerUrls = new ArrayList<CrawlerUrl>();
		if (doc != null) {
			Elements htmlLinks = doc.getElementsByTag("a");
			for (Element link : htmlLinks) {
				String href = link.attr("href");
				if (href != null && !href.trim().startsWith("http")) {
					href = getAbsoluteURL(currentCrawlerUrl.getUrl(), href);
				}
				if (null == href || href.trim().equals("") || href.trim().equals("#") || href.trim().equals("null"))
					continue;
				String title = link.text().trim();
				title = title == null || title.equals("") ? link.attr("title") : title;
				CrawlerUrl cUrl = new CrawlerUrl(href, title, childCrawlerListener);
				if (crawlerUrlFilter == null || (crawlerUrlFilter != null && crawlerUrlFilter.accept(cUrl))) {
					cUrl.setPolicy(Policy.DEPTH_FIRST);
					cUrl.setCurrentDepth((currentCrawlerUrl.getCurrentDepth() + 1));
					cUrl.setMeta(currentCrawlerUrl.getMeta());
					crawlerUrls.add(cUrl);
				}
			}
		}
		return crawlerUrls;
	}

	private static String getAbsoluteURL(String baseURI, String relativePath) {
		String abURL = null;
		try {
			URI base = new URI(baseURI);
			URI abs = base.resolve(relativePath);
			URL absURL = abs.toURL();
			abURL = absURL.toString();
		} catch (Exception e) {
		}
		return abURL;
	}

	public List<CrawlerUrl> getChildUrls(Element doc, CrawlerListener childCrawlerListener) throws Exception {
		return getChildUrls(doc, childCrawlerListener, null);
	}

	public String contentType() {
		return contentType;
	}

	public Date lastModified() {
		return lastModified;
	}

	public int length() {
		return length;
	}

	public Document document() {
		return document;
	}

	public String html() {
		return document.html();
	}
	public String trueUrl(){
		return document.baseUri();
	}

	public String title() {
		return document.title();
	}

	public String content() {
		if (content == null) {
			try{
				content=TextExtract.parse(html(),false);
			}catch(Exception e){
				content="";
			}
		}
		return content;
	}
}
